########################################################################
#  Pyfm - personal finance data-base
#  Copyright (C) 2007,8 Ehud Ben-Reuven
#  udi@benreuven.com
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation version 2.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
########################################################################
"""Flat files
All incoming statements are flat lines:
Statements have a relatively shallow and rigid structrue, in order to simplfy the processing
and make all formats of bank statements look the same, the statements are transformed
into lines that are completely flat:
each line contains all the information it needs without any reference to other lines.

A line is made from pairs of tag and value.
All the pairs in the same line are assumed to be bound togther and describe
different aspects of the same object(s) (bank, account, holding, transaction or asset).

Lines are used both to load information to the line data-base and to define queries on the data-base.
In a query it's possible to have a value which is itself another query (i.e. another flat line) so queries are not that flat after all.

The syntax of each line is:
<flat-line> = [SP <tag><op><value>  | SP <option> ]*
<option> = '-' [A-Z]+ # Options passed to the command shell through the line
<tag> = '_'? [A-Z] <any-word-character>*  # Tag that begins with '_' are usually not displayed
<op> = <command-op> | <relational-op>
<relational-op> = '=' | '!=' | '>=' | | '<='
<command-op> = '!' | '?' | '!?'
<value> = <text-without-blanks> | ...
        (QUOTE <any-text> QUOTE) | ...
        (<label>=QUOTE <any-text> QUOTE) | ... # The parser will remove the quotes
        '{' <flat-line> '}' | # for defining recursive queries: the value will be replaced with objects of class <tag>
                                # that are found in lines that match the query/s defined by the flat-lines surronded by ellipses
        # You can replace the regular casting of values into the class with the same class name as <tag>
        '/' <regular-expression> '/' | # cast to a regular expression
        '`' <python-expression> '`' | #  Evaluate the text surronded by a back-strich as a python expression and then cast it to <tag>
        '@' <another-tag-name> '@' # The entire text (including '@' at its start and end) will be used as value
                                                # This is used when creating 

* command-op are Used to define commands to the query mechanism
* The parse method converts lines to a dictionary in which the tag is the key and its value is a list of all the values the same tag received in the line.
As a result the order in which pairs appeared in the line is not preserved.
However if the <op> is different from '=' its value is prefixed to the tag before forming the key to the dictonary, as a result the same tag with
different <op> will have different keys in the dictonary.
* values will be casted from text strings (str) according to the tagtype_module passed as an argument to parse method
* Its possible to avoid the casting by tagtype_module by surronding the string with one of the following: "..." or {...} or /.../ or `...` or @...@

Tag names and the syntax of their value can be defined in other modules.

flat2objdb loads an entire flat file to DB:
None flat aspects of the file:
* All lines are assumed to come from the file even if the file name itself does not appear in each line (see FILE tag below)
this is trivial but it has some implications.
For example: two lines in the same file showing a transaction from the same holding on the same time
will be assumed to be two different transactions if they come from the same file, however the same two lines
coming from two different files are more likely to be two different discriptions of the same transaction.
Note that the file name itself will be used as a reference, without its path or extension or any other
signature so different unique names should be used for different flat files.
"""
# TODO Add \ to flat lines parsing (flat.parse)
# FIXME "TAG={}" returns {'TAG':[] } do we want this? dont we want {'TAG':[{}]}


import os
import re
import sys
from retype import RE
from copy import copy

reobj=re.compile(r"""(?x) # allow for commenting inside the RE
\s* # Ignore white space
(
(?P<comment>\#) # Ignore the rest of the line.
|(?P<endcurl>}) # Look for end of a recursive call to parse.
|(?P<option>\-[A-Z]+(?=(\s|$))) # or look for an option
|( # or look for a regular tag-value pair
   (?P<tag>[\_A-Z]\w*) # tag
   (?P<op>(# look for operator
      ((\!)?(&)?((<=)|(>=)|=|<|>))| # The relation operation between tag and value
      ((\!\?)|(\?))| # Check if tag exists in a line without specifing a value
      (\!(?=[A-Z])) # Command of the form <tag>!<command> must begin with a capital letter
   )) # end of operator
   ( # Value
      (?P<startcurl>{)| # look for  a start of a recursion call
      ( # or look for a regular value
         (?P<label>[A-Za-z]{2,6}=(?="))? # Break into label=value pair only if the value is quoted
         (?P<quote>"|/|`|@|\|)? # look for quote at start of value
         (?P<value>.*?) # finally the value. Dont be greedy so not to eat the next token. Note that an empty value is also OK
         (?(quote)
            ((?<!\\)(?P=quote))|  # If there was a quote at the start then look for it at the end. Skip backslash quote
            (?=(}|\s|$))) # If there wasn't a quote at start: The value is not greedy so force an unquoted value to be followed with:
                                # white-space or an end-curl or to continue to end of string
                                # Note: [] did not worked 
      )
   ) #end of value
) # end of tag-value pair
)""")

tagtype_syntax=None
def parse(line,tagtype_module=None):
        """Parse a line from a flat file into its tag-value pairs.
        The return-value is a dictionary where the keys are the tags and the values are the list
        of the values that appeared in the line.
        
        If tagtype_module is given then the tag name will be looked for in it and if found
        its method will be used to cast the string value in l. If the tag is _LINE_INDEX_ then it will
        be cated to int.
        
        The same tag can appear several times in a line and all its values will appear in a list.
        The syntax of each pair is of the form tag=value or >=, <=, <, > or != or /.
        The operation is prefixed to the tag (except for = or /)

        Surounding quotes on values are removed
        &quot; is translated back to a quote char.
        
        The value can be surrounded by {} in which case it will be sent recursivly to parse
        and the value will be the returned dictionary.
        """
        if not tagtype_module: tagtype_module=tagtype_syntax # Can't put a simple default assigment in function signature because tagtype_syntax is set latter
        if not line: return {}
        # line may be list. In this case on entry it sould contain the text line to be parsed
        # as a first argument and on return it will have the unparsed part of the text line in the
        # first element.
        if isinstance(line,list):
                l=line[0]
        else:
                l=line
        ldict={}
        while True:
                p=reobj.match(l)
                if not p: break
                
                # handle the return value in line for a recursive usage
                l=l[p.end():]
                if isinstance(line,list): line[0]=l
                if p.group('comment'):
                        return ldict
                if p.group('endcurl'):
                        if not isinstance(line,list): raise Exception('Unbalanced }')
                        return ldict
                if p.group('option'):
                        ldict[p.group('option')]=[]
                        continue
                
                tag=p.group('tag')
                raw_tag=tag
                op=p.group('op')
                if op!='=':
                        tag = op + tag
                values=ldict.setdefault(tag,[]) # Make sure the tag appears in ldict even if there is no value.
                quote=p.group('quote')
                if p.group('startcurl'):
                        # recursive call
                        # Pass to parse the line surronded by list so we can get it back after the end of the recursive parsing
                        u=[l]
                        values.append(parse(u,tagtype_module))
                        # Update the line beeing parsed (l) to its new location after the recusive parsing just done.
                        l=u[0]
                        # Make sure the caller of this parse will also receive the new location
                        if isinstance(line,list): line[0]=l
                else:
                        value=p.group('value')
                        value=value.replace('&quot;','"')
                        # Restore labeld value to its original form but without the quotes
                        if p.group('label'): value = p.group('label') + value # label includes = at its end
                        # Cast values in line
                        if not value: # empty values don't need to be added to list of values for tag in ldict
                                pass
                        elif op in ['!','!?','?']: # Commands dont need casting
                                values.append(value)
                        elif quote=='@': # A value that is surronded with a @ is an indication of a reference tag
                                values.append(quote+value+quote)
                        elif quote=='/': # Built in casting
                                # A value that is surronded with / is casted to be a regular expression
                                values.append(RE(value))
                        elif quote=='`':
                                values.append(eval(value))
                        elif quote=='|':
                                import obj # This needs to be here because obj uses this module
                                value=parse(value,tagtype_module)
                                filter={}
                                if raw_tag.endswith('FILTER'):
                                        build_tag=raw_tag[:-len('FILTER')]
                                else:
                                        build_tag=raw_tag
                                obj.db.build_filter(build_tag,value,filter)
                                if filter:  values.extend(filter[build_tag])
                        elif raw_tag in ['_LINE_INDEX_']: # Build in casting
                                values.append(int(value))
                        elif tagtype_module:
                                # Cast values of a ldict[tag] from a strings to the tag's class defined in tagtype_module
                                if raw_tag in dir(tagtype_module):
                                        values.append(getattr(tagtype_module,raw_tag)(value))
                                else:
                                        values.append(value)
                        else:
                                values.append(value)
        return ldict

def line_copy(l):
        l=l.copy()
        for t in l:
                l[t]=copy(l[t])
                for i,v in enumerate(l[t]):
                        if isinstance(v,dict):
                                l[t][i]=line_copy(v)
        return l

def line_merge(l1,l2):
        """Update l1 with the content of l2.
Return true if l1 was modified.
"""
        merge=False
        for tag in l2:
                v1s=l1.get(tag,[])
                for v in l2[tag]:
                        if v not in v1s:
                                merge=True
                                v1s.append(v)
                l1[tag]=v1s
        return merge

def squote(val):
        val=val.replace('"','&quot;') # TODO: check quopri
        if val.find(' ')!=-1:
                return '"' + val + '"'
        else:
                return val
        
def break_tag(tag):
        """Decompose a tag into a 3-tuple made from the tag as it appeared in the filter,
        the real (raw) tag name, and the operator used between the raw tag and its value"""
        c=0
        while tag[c] in '!&<>=?': c+=1
        if c:
                return (tag,tag[c:],tag[:c])
        else:
                return (tag,tag,'=')

def stag(tag):
        if tag:
                (t,r,o)=break_tag(tag)
                return r+o
        else:
                return ''

def sobj(obj,parent=None):
        """Return a string that uniquely represnts the object
as a <tag>=<filter> pair where the filter is in :...: format."""
        if isinstance(obj,dict):
                return 'LINE='+sdict(obj,'{}',parent)
        line={}
        obj.build_line(line)
        return stag(obj.classname)+sdict(line,'||',parent)

def sdict(value,filterquote=None,parent=None):
        s=[]
        for t in value:
                s.append(stagvalue(t,value[t],parent))
        if filterquote:
                return filterquote[0]+' '.join(s)+filterquote[1]
        else:
                return ' '.join(s)

def stv(obj,tag):
        return stagvalue(tag,obj[tag])

def stagvalue(tag,value,parent=None,short=False):
        """Use the short flag to indicate that you want the <tag>= prefix to be placed only once at the start"""
        #Avoid loops
        if id(value)==id(parent): return ''
        if not parent: parent=value
        
        if value==None: return '' # It may be 0
        if isinstance(value,basestring):
                return stag(tag)+squote(value)
        elif isinstance(value,list):
                s=[]
                if short and len(value)>1:
                        tag1=stag(tag)
                        tag=''
                else:
                        tag1=''
                for v in value:
                        s.append(stagvalue(tag,v,parent))
                return tag1+' '.join(s)
        elif isinstance(value,dict):
                return stag(tag)+sdict(value,'{}',parent)


        try:
                if value.classname==tag:
                        return sobj(value,parent)
        except AttributeError:
                pass
                
        return stag(tag)+squote(`value`)

def ptagvalue(tag,value,fout=sys.stdout):
        if value:
                print >>fout,stagvalue(tag,value),

def pacctid(acctid,fout=sys.stdout):
        ptagvalue("ACCTID",acctid,fout)
                
def psecid(secid,fout=sys.stdout):
        ptagvalue("SECID",secid,fout)

def flat2objdb(fname,objdb,tagtype_module=None):
        """Load a file to database objdb.
        For each tag in each line, look for a class with the same name as the tag in the module passed
        through tagtype_module and use the class definition to cast the values in the line.
        """
        if not tagtype_module: tagtype_module=tagtype_syntax # Can't put a simple default assigment in function signature because tagtype_syntax is set latter
        # TODO: add handling of \ to allow multiple lines in the file to be parsed as one line.
        file=os.path.splitext(os.path.split(fname)[1])[0]
        fin = open(fname,'rb') # The 'b' is needed because asset names can have unicode values
        nline=1
        for l in fin:
                line=parse(l,tagtype_module)
                if line:
                        # Add implicit information to each line
                        if 'FILE' not in line: line['FILE']=[file]
                        if 'NLINE' not in line: line['NLINE']=[file +'#'+str(nline)]
                        # Update database with line info
                        objdb.line_update(line)
                nline+=1
        fin.close()
                

def sprint_line(r,tags):
        s=[]
        for t in tags:
            ptag=t
            for v in r.get(t,[]):
                s.append(stagvalue(ptag,v))
                # if a line contains only one tag then print it only once.
                if len(tags)==1: ptag=''
        return ' '.join(s)

def print_line(r,expand=False,tags=None):
    if not tags: tags=r.keys()
    # If there is a single result,
    if expand:
        # use a seperate line for each tag
        # and print the tag only at the start of the line
        for t in tags:
            ptag=t
            s=[]
            for v in r.get(t,[]):
                s.append(stagvalue(ptag,v))
                ptag=None
            if s: print ' '.join(s)
    else:
        s=sprint_line(r,tags)
        if s: print s,


def add2line(line,tag,value=None):
        if isinstance(value,list):
                if not value:
                        add2line(line,tag)
                else:
                        for v in value:
                                add2line(line,tag,v)
                return
        if tag not in line:
                if value!=None:
                        line[tag]=[value]
                else:
                        line[tag]=[]
        elif value not in line[tag]:
                if value!=None:
                        line[tag].append(value)

if __name__=='__main__':
        print parse('A!PRINT A=B G? C={D=1 E={Z=12 B=1}} F!?')
